# 要爬取的URL列表
import csv
import os
import time

import requests
import os
from concurrent.futures import ThreadPoolExecutor
from fake_useragent import UserAgent
import pymongo

import requests
ua = UserAgent()
url = 'https://www.governbond.org.cn:4443/api/loadBondData.action?timeStamp=1744284092347&dataType=ZQFXLISTBYAD&adList=&adCode=87&zqlx=&year=&fxfs=&qxr=&fxqx=&zqCode=&zqName=&page=1&pageSize=10'
paramss = []
for i in range(1, 1642):
    params = {
        'dataType': 'ZQFXLISTBYAD',
        'adCode': '87',
        'page': f'{i}',
        'pageSize': '10',
    }
    paramss.append(params)


# 爬取单个页面的函数
def fetch_url(params):  # 谁在列表里 fetch_url(params)，谁就是参数


    try:
        response = requests.get(url, params=params, headers={'User-Agent': ua.random})
        print(response.json())
        data = []
        for item in response.json()['data']:
            print(f'正在爬取第{item['ZQ_NAME']}页')
            ZQ_NAME = item['ZQ_NAME']
            ZQ_CODE = item['ZQ_CODE']
            AD_NAME = item['AD_NAME']
            ZQ_FXTIME = item['ZQ_FXTIME']
            ZQQX_NAME = item['ZQQX_NAME'].strip()
            LXFS_ID = item['LXFS_ID']
            LL = item['LL']
            FXFS = item['FXFS']
            QX_DATE = item['QX_DATE']
            data.append([ZQ_NAME, ZQ_CODE, AD_NAME, ZQ_FXTIME, ZQQX_NAME, LXFS_ID, LL, FXFS, QX_DATE])
            time.sleep(1)

            # print(ZQ_NAME,ZQ_CODE,AD_NAME,ZQ_FXTIME,ZQQX_NAME,LXFS_ID,LL,FXFS,QX_DATE)
            with open('地方债.csv', 'w', encoding='utf-8', newline='') as f:
                writer = csv.writer(f)
                writer.writerow(['债卷名', '债卷代码', '地区', '发行时间', '发行期限', '利率', '付息方式', '到期日期'])
                writer.writerows(data)
    except Exception as e:
        print(f"Error fetching {url}: {e}")


# 使用线程池并发爬取
def main(urls, max_workers=12):
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        # 提交任务给线程池  url是传递给fetch_url函数的参数
        futures = [executor.submit(fetch_url, param) for param in
                   paramss]  # executor.submit() 接收的是一个函数引用，它会在合适的时间（即线程池空闲时）调用该函数
        # 等待所有任务完成
        # for future in futures:
        #     try:
        #         # 调用 result() 方法获取任务执行的结果
        #         future.result()  # result() 是 Future 类的一个方法，用来获取任务的返回值。当你调用 result() 时，程序会阻塞，直到任务完成，然后返回任务的执行结果
        #     except Exception as e:
        #         print(f"任务出错: {e}")


# 运行线程池爬虫
if __name__ == "__main__":
    main(paramss, max_workers=os.cpu_count())  # 控制线程池中最大并发线程数为3 它返回系统中可用的 CPU 核心数量。

